from transformers import AutoTokenizer
import json
import os

def convert_tokenizer_model_to_json(model_path, output_path=None):
    """
    Convert a tokenizer.model file to tokenizer.json format.

    Args:
        model_path: Path to the tokenizer.model file or directory containing it
        output_path: Optional output path for tokenizer.json (defaults to same directory)
    """
    # Handle both file and directory paths
    if os.path.isdir(model_path):
        tokenizer_model_path = os.path.join(model_path, "tokenizer.model")
    else:
        tokenizer_model_path = model_path
        model_path = os.path.dirname(model_path)

    if not os.path.exists(tokenizer_model_path):
        raise FileNotFoundError(f"tokenizer.model not found at {tokenizer_model_path}")

    tokenizer = AutoTokenizer.from_pretrained(model_path)

    if output_path is None:
        output_path = model_path

    tokenizer.save_pretrained(output_path)

    tokenizer_json_path = os.path.join(output_path, "tokenizer.json")
    if os.path.exists(tokenizer_json_path):
        print(f"Successfully created tokenizer.json at {tokenizer_json_path}")
    else:
        print("Warning: tokenizer.json was not created. The tokenizer might not support this format.")

    return tokenizer_json_path

# Example usage
if __name__ == "__main__":
    # Example: Convert a tokenizer.model file
    # convert_tokenizer_model_to_json("/path/to/tokenizer.model")

    # Example: Convert from a directory containing tokenizer.model
    # convert_tokenizer_model_to_json("/path/to/model/directory")